package org.example.day20241227.operators;

import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.connector.file.src.FileSource;
import org.apache.flink.connector.file.src.FileSourceSplit;
import org.apache.flink.connector.file.src.impl.StreamFormatAdapter;
import org.apache.flink.connector.file.src.reader.BulkFormat;
import org.apache.flink.connector.file.src.reader.TextLineInputFormat;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;
import org.apache.flink.util.Collector;

import java.time.Duration;


/**
 * @className: FlatMapDemo2
 * @author: 孙磊
 * @date: 2024/12/29 18:01
 * @Version: 1.0
 * @description: 输入一个元素同时产生零个、一个或多个元素。
 * 1.2.flatmap算子,读一个文本数据,生成单词列表,得到一个新的数据集;
 */
public class FlatMapDemo {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        env.setParallelism(1);
        BulkFormat<String, FileSourceSplit> bulkFormat =
                new StreamFormatAdapter<>(new TextLineInputFormat());
        //forRecordStreamFormat:从文件流中读取文件内容
        //forBulkFileFormat:从文件中一次读取一批记录

        FileSource<String> source = FileSource.forBulkFileFormat(
                bulkFormat,
                new Path("src/main/resources/words.txt")).build();

        SingleOutputStreamOperator<String> flatMap = env.fromSource(source, WatermarkStrategy.noWatermarks(), "fileSource")
                .flatMap(new FlatMapFunction<String, String>() {
                    @Override
                    public void flatMap(String s, Collector<String> out) throws Exception {
                        String[] words = s.split(" ");
                        for (String word : words) {
                            out.collect(word);
                        }
                    }
                });
        //将处理过的数据再次写入到文件中
        Path path = new Path("src/main/resources");
        final FileSink<String> sink = FileSink
                .forRowFormat(path, new SimpleStringEncoder<String>("UTF-8"))
                .withRollingPolicy(
                        DefaultRollingPolicy.builder()
                                .withRolloverInterval(Duration.ofMinutes(15))
                                .withInactivityInterval(Duration.ofMinutes(5))
                                .withMaxPartSize(MemorySize.ofMebiBytes(1024))
                                .build())
                .build();
        flatMap.sinkTo(sink);

        env.execute();

    }
}
